﻿using System;
using System.Collections.Generic;
using System.Linq;
using System.Text;
using System.Configuration;
using System.IO;
using System.Net;
using System.IO.Compression;
using SEO.Entity;
using System.Xml;
using SEO.Business;

namespace WebCrawler
{
    /// <summary>
    /// 工具类
    /// </summary>
    public static class Common
    {
        #region 登记日志
        /// <summary>
        /// 登记日志
        /// </summary>
        /// <param name="info"></param>
        public static void Log(string info)
        {
            try
            {
                string file = ConfigurationManager.AppSettings["LogFile"] + "\\" + DateTime.Now.ToString("yyyy");
                if (!Directory.Exists(file))
                {
                    Directory.CreateDirectory(file);
                }
                file += "\\" + DateTime.Now.ToString("yyyyMMdd") + ".log";
                StreamWriter sw = new StreamWriter(file, true, System.Text.Encoding.UTF8);
                sw.WriteLine(string.Format("{0} --- {1}", DateTime.Now.ToString("yyyy-MM-dd HH:mm:ss"), info));
                sw.Close();
            }
            catch { }
        }
        #endregion

        /// <summary>
        /// 获取Url
        /// </summary>
        /// <param name="strBaiDuUrl"></param>
        /// <returns></returns>
        public static string GetFirstUrl(string strBaiDuUrl)
        {
            if (string.IsNullOrEmpty(strBaiDuUrl))
                return string.Empty;
            try
            {
                HttpWebRequest request = (HttpWebRequest)HttpWebRequest.Create(strBaiDuUrl);
                request.AllowAutoRedirect = false;
                request.Referer = strBaiDuUrl;
                string strUrl = string.Empty;
                using (WebResponse response = request.GetResponse())
                {
                    strUrl = response.Headers["Location"].Trim();
                }
                if (string.IsNullOrEmpty(strUrl))
                {
                    strUrl = GetUrl(strBaiDuUrl);
                }
                //去"/"
                if (strUrl.Substring(strUrl.Length - 1, 1) == "/")
                {
                    strUrl = strUrl.Substring(0, strUrl.Length - 1);
                }              
                return strUrl;
            }
            catch (Exception ex)
            {
                Log("获取百度跳转真实页面失败:" + strBaiDuUrl + " " + ex.Message.ToString());
                return "";
            }
        }


        #region 获取跳转后的url
        /// <summary>
        /// 获取跳转后的url
        /// </summary>
        /// <param name="strBaiDuUrl"></param>
        /// <returns></returns>
        public static string GetUrl(string strBaiDuUrl)
        {
            try
            {
                WebRequest requestPage = WebRequest.Create(strBaiDuUrl);
                HttpWebResponse responsePage = (HttpWebResponse)requestPage.GetResponse();
                string strUrl = string.Empty;
                if (responsePage.StatusCode == HttpStatusCode.OK)
                {
                    strUrl = responsePage.ResponseUri.AbsoluteUri.Trim();
                    //去"/"
                    if (strUrl.Substring(strUrl.Length - 1, 1) == "/")
                    {
                        strUrl = strUrl.Substring(0, strUrl.Length - 1);
                    }
                }
                responsePage.Close();
                return strUrl;
            }
            catch (Exception ex)
            {
                Log("获取百度跳转页面失败:" + strBaiDuUrl + " " + ex.Message.ToString());
                return "";
            }
        }
        #endregion

        /// <summary>
        /// Url编码
        /// </summary>
        /// <param name="str"></param>
        /// <returns></returns>
        public static string UrlEncode(string str)
        {
            StringBuilder sb = new StringBuilder();
            byte[] byStr = System.Text.Encoding.UTF8.GetBytes(str);
            for (int i = 0; i < byStr.Length; i++)
            {
                sb.Append(@"%" + Convert.ToString(byStr[i], 16));
            }

            return (sb.ToString());
        }

        #region 获取页面HTML
        /// <summary>
        /// 获取页面HTML
        /// </summary>
        /// <param name="strUrl">路径</param>
        /// <returns></returns>
        public static string GetPageHTML(string strUrl)
        {
            string strHTML = string.Empty;
            try
            {

                WebRequest requestPage = WebRequest.Create(strUrl);
                HttpWebResponse responsePage = (HttpWebResponse)requestPage.GetResponse();
                Stream streamReceive = responsePage.GetResponseStream();
                string strEncod = responsePage.CharacterSet;
                if (strEncod == null || strEncod.Length == 0) strEncod = "utf-8";
                if (strEncod.ToLower() == "iso-8859-1") strEncod = "utf-8";
                if (responsePage.ContentEncoding.ToLower() == "gzip")
                {
                    streamReceive = new GZipStream(streamReceive, CompressionMode.Decompress);
                }
                using (StreamReader readerPage = new StreamReader(streamReceive, Encoding.GetEncoding(strEncod)))
                {
                    strHTML = readerPage.ReadToEnd();
                    readerPage.Dispose();
                }
                streamReceive.Dispose();
                responsePage.Close();
            }
            catch (Exception ex)
            {
                Log("抓取页面出错:" + strUrl + " " + ex.Message.ToString());
            }
            return strHTML;
        }
        #endregion

        /// <summary>
        /// 获取截止日期
        /// </summary>
        /// <returns></returns>
        public static DateTime GetAccountDay(SysConfig info)
        {
            DateTime dDateTime = DateTime.Now;
            int nMonth = 0;
            int nYear = dDateTime.Year;
            if (dDateTime.Day < info.AccountDay.Value)
            {
                nMonth = dDateTime.AddMonths(-1).Month;
            }
            else
            {
                nMonth = dDateTime.Month;
            }
            if (nMonth == 12 && dDateTime.Month == 1)
                nYear--;
            return Convert.ToDateTime(string.Format("{0}-{1}-{2}", nYear, nMonth, info.AccountDay.Value));
        }

        /// <summary>
        /// 获取结算时间
        /// </summary>
        /// <param name="dBeginTime"></param>
        /// <param name="nAccountDay"></param>
        /// <returns></returns>
        public static DateTime GetEndTime(DateTime dBeginTime, int nAccountDay)
        {
            int nMonth = 0;
            int nYear = dBeginTime.Year;
            if (dBeginTime.Day < nAccountDay)
            {
                nMonth = dBeginTime.Month;
            }
            else
            {
                nMonth = dBeginTime.AddMonths(1).Month;
            }
            if (dBeginTime.Month == 12 && nMonth == 1)
                nYear++;
            return Convert.ToDateTime(string.Format("{0}-{1}-{2}", nYear, nMonth, nAccountDay));
        }

        #region 获取域名后缀
        public static List<string> DomainList = null;
        /// <summary>
        /// 获取域名后缀列表
        /// </summary>
        /// <returns></returns>
        public static List<string> GetDomainList()
        {
            if (DomainList == null)
            {
                DomainList = new List<string>();
                try
                {
                    XmlDocument xmlDoc = new XmlDocument();
                    xmlDoc.Load(ConfigurationManager.AppSettings["DomainXml"]);
                    XmlNodeList nodeList = xmlDoc.SelectNodes("root/tldns/tldn");
                    foreach (XmlNode nodeItem in nodeList)
                    {
                        DomainList.Add(nodeItem.InnerText.Trim());
                    }
                }
                catch (Exception ex)
                {
                    Log("加载域名后缀xml出错：" + ex.Message);
                }
            }
            return DomainList;
        }

        /// <summary>
        /// 根据域名获取顶级域名
        /// </summary>
        /// <param name="strDomain"></param>
        /// <returns></returns>
        public static string GetTopDomain(string strDomain)
        {
            string lowerDomainName = strDomain.ToLower();
            if (lowerDomainName.IndexOf("http://") == 0)
                lowerDomainName = lowerDomainName.Substring(7, lowerDomainName.Length - 7);
            List<string> SelectDomainList = GetDomainList();
            if (SelectDomainList != null)
            {
                string[] temPart = lowerDomainName.Split('/');
                if (temPart.Length > 0)
                {
                    lowerDomainName = temPart[0];
                    int index = lowerDomainName.IndexOf('.');
                    string tempTopDomainName = lowerDomainName;
                    string domainPart = String.Empty;

                    if (index > 0)
                    {
                        bool bTag = false;
                        do
                        {
                            tempTopDomainName = tempTopDomainName.Substring(index, tempTopDomainName.Length - index);
                            foreach (string strSelectDomain in SelectDomainList)
                            {
                                if (tempTopDomainName == strSelectDomain)
                                {
                                    domainPart = strSelectDomain;
                                    bTag = true;
                                    break;
                                }
                            }
                            if (bTag)
                                break;
                            tempTopDomainName = tempTopDomainName.Substring(1, tempTopDomainName.Length - 1);
                            index = tempTopDomainName.IndexOf('.');
                        } while (index > 0);
                        if (!String.IsNullOrEmpty(domainPart))
                        {
                            string namePart = lowerDomainName.Remove(lowerDomainName.Length - domainPart.Length);
                            index = namePart.LastIndexOf('.');
                            if (index > 0)
                            {
                                namePart = namePart.Substring(index + 1, namePart.Length - index - 1);
                            }
                            return namePart + domainPart;
                        }
                    }
                }
            }
            return lowerDomainName;
        }
        #endregion

        /// <summary>
        /// 根据域名获取顶级域名
        /// </summary>
        /// <param name="strDomain"></param>
        /// <returns></returns>
        public static string GetLastDomain(string strDomain)
        {
            string lowerDomainName = strDomain.ToLower();
            if (lowerDomainName.IndexOf("http://") == 0)
                lowerDomainName = lowerDomainName.Substring(7, lowerDomainName.Length - 7);
            string[] temPart = lowerDomainName.Split('/');
            lowerDomainName = temPart[0];
            if (lowerDomainName.IndexOf("wwww.") == 0)
                lowerDomainName = lowerDomainName.Substring(5, lowerDomainName.Length - 5);
            return lowerDomainName;
        }

        /// <summary>
        /// 验证网址是否新闻源
        /// </summary>
        /// <param name="strUrl"></param>
        /// <returns></returns>
        public static int CheckNewsWeb(string strUrl)
        {
            //获取文章类型
            BisNewsWeb bisInfo = new BisNewsWeb();
            string strTopDomain = Common.GetTopDomain(strUrl);
            int nAimID = bisInfo.GetNewsWebID(strTopDomain, true);
            if (nAimID == 0)
            {
                if (strUrl.ToLower().IndexOf("http://wwww.") < 0)
                {
                    strTopDomain = Common.GetLastDomain(strUrl);
                    nAimID = bisInfo.GetNewsWebID(strTopDomain, true);
                }
            }
            return nAimID;
        }
    }
}
